package com.leilei;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.operators.Order;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.operators.UnsortedGrouping;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;

import java.util.Arrays;
import java.util.List;


/**
 * @author lei
 * @version 1.0
 * @date 2021/3/7 14:08
 * @desc 使用flink 的DataSet类型完成对单词计数
 * 编码步骤
 * 1.准备环境-env
 * 2.准备数据-source
 * 3.处理数据-transformation
 * 4.输出结果-sink
 * 5.触发执行- execute  如果有print,DataSet不需要调用execute,DataStream需要调用execute
 */
public class WordCountDataSet {
    public static void main(String[] args) throws Exception {
        //1. 准备环境
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

        //2. 准备数据源
        //  DataSource<String> elements 为实际类型 但下方用了DataSet（父类）接受
        DataSet<String> elements = env.fromElements("java,scala,php,c++", "java,scala,php", "java,scala", "java");

        //3. 处理数据
        //3.1 每一行数据按照,切分成一个个的单词组成一个集合
        //  FlatMapOperator<String, String> 为实际类型 但下方用了DataSet（父类）接受
        DataSet<String> flatMap = elements.flatMap((String element, Collector<String> out) -> Arrays.stream(element.split(","))
                .forEach(out::collect))
                .returns(Types.STRING);
        // 3.2  对集合中的每个单词记为1
        //  MapOperator<String, Tuple2<String, String>> 为实际类型 但下方用了DataSet（父类）接受
        DataSet<Tuple2<String, Integer>> mapOperator = flatMap.map
                (word -> Tuple2.of(word, 1))
                .returns(Types.TUPLE(Types.STRING,Types.INT));
        // 3.3 分组
        // 按照 Tuple2<String, Integer> 字段0（key）分组,上一步mapOperator 操做后 Tuple2<单词, 1> ，0位置则为具体单词 1位置则为单词数量
        UnsortedGrouping<Tuple2<String, Integer>> group = mapOperator.groupBy(0);
        // 对 Tuple2<String, Integer> 字段1(value)求和
        // AggregateOperator<Tuple2<String, Integer>>
        // 3.4 求和
        DataSet<Tuple2<String, Integer>> sum = group.sum(1);
        // 3.5 排序 setParallelism 是设置并行度
        // SortPartitionOperator 为实际类型 但下方用了DataSet（父类）接受
        DataSet<Tuple2<String, Integer>> result = sum.
                sortPartition(1, Order.DESCENDING)
                .setParallelism(3);
        // 4. 输出结果-sink 可选择多种放肆   result.collect() 、result.print()、result.count();
        // [(php,2), (scala,3), (c++,1), (java,4)]
        List<Tuple2<String, Integer>> collect = result.collect();
        result.print();
    }
}
